Configurable buffer sync

diff --git a/ChangeLog-2.0.30 b/ChangeLog-2.0.30
index 0c3977838ee9439c0c8ddcb620b9a10185005256..1e59c3ce8f57954f07c95535b0c559ed5d932149 100644 (file)
--- a/ChangeLog-2.0.30
+++ b/ChangeLog-2.0.30
@@ -3,3 +3,7 @@
   * Removed deprecated info- and loaddb requests.
   
   * Added named pipe support. (#104, @pavelxdd)
+
+  * Changed writing buffer length to reduce random io.
+  
+  * Added option to fsync the buffer on a configurable interval.
\ No newline at end of file

diff --git a/Debug/src/siri/subdir.mk b/Debug/src/siri/subdir.mk
index 9c8a4a5cd669ceca0cd84d83d25110b66aec1198..9f48e2f15f0eedfb2ede1b22013bbd64a253fd38 100644 (file)
--- a/Debug/src/siri/subdir.mk
+++ b/Debug/src/siri/subdir.mk
@@ -6,6 +6,7 @@
 C_SRCS += \
 ../src/siri/async.c \
 ../src/siri/backup.c \
+../src/siri/buffersync.c \
 ../src/siri/err.c \
 ../src/siri/heartbeat.c \
 ../src/siri/optimize.c \
@@ -15,6 +16,7 @@ C_SRCS += \
 OBJS += \
 ./src/siri/async.o \
 ./src/siri/backup.o \
+./src/siri/buffersync.o \
 ./src/siri/err.o \
 ./src/siri/heartbeat.o \
 ./src/siri/optimize.o \
@@ -24,6 +26,7 @@ OBJS += \
 C_DEPS += \
 ./src/siri/async.d \
 ./src/siri/backup.d \
+./src/siri/buffersync.d \
 ./src/siri/err.d \
 ./src/siri/heartbeat.d \
 ./src/siri/optimize.d \

diff --git a/Release/src/siri/subdir.mk b/Release/src/siri/subdir.mk
index c65a15d89dafc898697744e52fd02ce3bbf0d265..a506e6e8037b612f0d69b957720a66586095a260 100644 (file)
--- a/Release/src/siri/subdir.mk
+++ b/Release/src/siri/subdir.mk
@@ -6,6 +6,7 @@
 C_SRCS += \
 ../src/siri/async.c \
 ../src/siri/backup.c \
+../src/siri/buffersync.c \
 ../src/siri/err.c \
 ../src/siri/heartbeat.c \
 ../src/siri/optimize.c \
@@ -15,6 +16,7 @@ C_SRCS += \
 OBJS += \
 ./src/siri/async.o \
 ./src/siri/backup.o \
+./src/siri/buffersync.o \
 ./src/siri/err.o \
 ./src/siri/heartbeat.o \
 ./src/siri/optimize.o \
@@ -24,6 +26,7 @@ OBJS += \
 C_DEPS += \
 ./src/siri/async.d \
 ./src/siri/backup.d \
+./src/siri/buffersync.d \
 ./src/siri/err.d \
 ./src/siri/heartbeat.d \
 ./src/siri/optimize.d \

diff --git a/include/siri/buffersync.h b/include/siri/buffersync.h
new file mode 100644 (file)
index 0000000..ebc8bd6
--- /dev/null
+++ b/include/siri/buffersync.h
@@ -0,0 +1,17 @@
+/*
+ * buffersync.h - Buffer sync.
+ *
+ * author       : Jeroen van der Heijden
+ * email        : jeroen@transceptor.technology
+ * copyright    : 2018, Transceptor Technology
+ *
+ */
+#ifndef SIRI_BUFFERSYNC_H_
+#define SIRI_BUFFERSYNC_H_
+
+#include <siri/siri.h>
+
+void siri_buffersync_init(siri_t * siri);
+void siri_buffersync_stop(siri_t * siri);
+
+#endif  /* SIRI_HEARTBEAT_H_ */

diff --git a/include/siri/cfg/cfg.h b/include/siri/cfg/cfg.h
index 4cc44b192be5dbb196c37075d0f1fdc9ebfa4bff..0342b688e8acf38c238ddf298398a2f1400b37a0 100644 (file)
--- a/include/siri/cfg/cfg.h
+++ b/include/siri/cfg/cfg.h
@@ -38,6 +38,7 @@ struct siri_cfg_s
     char default_db_path[XPATH_MAX];
     uint8_t pipe_support;
     char pipe_client_name[XPATH_MAX];
+    uint32_t buffer_sync_interval;
 };
 
 #endif  /* SIRI_CFG_H_ */

diff --git a/include/siri/siri.h b/include/siri/siri.h
index 705477703127177fccf4084c9407a26b8a6438ec..ece3af80480ce00dd9b929b54413af82c4af2689 100644 (file)
--- a/include/siri/siri.h
+++ b/include/siri/siri.h
@@ -56,6 +56,7 @@ struct siri_s
     siri_optimize_t * optimize;
     uv_timer_t * backup;
     uv_timer_t * heartbeat;
+    uv_timer_t * buffersync;
     siri_cfg_t * cfg;
     siri_args_t * args;
     uv_mutex_t siridb_mutex;

diff --git a/itest/testing/server.py b/itest/testing/server.py
index 96edb0994eedd50a221a27f0f2de068b0d0c616c..296d37178111ba3d6343c38b620ffac2ce10addf 100644 (file)
--- a/itest/testing/server.py
+++ b/itest/testing/server.py
@@ -31,8 +31,9 @@ class Server:
 
     def __init__(self,
                  n,
-                 optimize_interval=30,
+                 optimize_interval=300,
                  heartbeat_interval=30,
+                 buffer_sync_interval=0,
                  compression=True,
                  pipe_name=None,
                  **unused):
@@ -44,6 +45,7 @@ class Server:
             pipe_name
         self.listen_client_port = 9000 + n
         self.listen_backend_port = 9010 + n
+        self.buffer_sync_interval = buffer_sync_interval
         self._server_address = self.SERVER_ADDRESS
         self.server_address = \
             self._server_address.lstrip('[').rstrip(']').replace(
@@ -76,6 +78,7 @@ class Server:
         config.set('siridb', 'ip_support', self.ip_support)
         config.set('siridb', 'optimize_interval', self.optimize_interval)
         config.set('siridb', 'heartbeat_interval', self.heartbeat_interval)
+        config.set('siridb', 'buffer_sync_interval', self.buffer_sync_interval)
         config.set('siridb', 'default_db_path', self.dbpath)
         config.set('siridb', 'max_open_files', MAX_OPEN_FILES)
         config.set('siridb', 'enable_shard_compression', int(self.compression))

diff --git a/siridb.conf b/siridb.conf
index 7e6c665fa9d61b1e6beebcde4f9c60547153b185..689e11a47ca55413dda1f50fe89b01edc254a248 100644 (file)
--- a/siridb.conf
+++ b/siridb.conf
@@ -52,6 +52,15 @@ optimize_interval = 3600
 #
 heartbeat_interval = 30
 
+#
+# SiriDB can run fsync on the buffer file on an interval in milliseconds. 
+# This value is set to 0 by default which tells SiriDB to run fsync after 
+# each insert request. When having many insert requests per second, it can be 
+# useful to use an interval like 500 milliseconds.
+#
+#buffer_sync_interval = 500
+buffer_sync_interval = 0
+
 #
 # SiriDB will not open more shard files than max_open_files. Note that the
 # total number of open files can be sligtly higher since SiriDB also needs

diff --git a/src/siri/buffersync.c b/src/siri/buffersync.c
new file mode 100644 (file)
index 0000000..e6723db
--- /dev/null
+++ b/src/siri/buffersync.c
@@ -0,0 +1,71 @@
+/*
+ * buffersync.c - Buffer sync.
+ *
+ * author       : Jeroen van der Heijden
+ * email        : jeroen@transceptor.technology
+ * copyright    : 2018, Transceptor Technology
+ *
+ */
+#include <logger/logger.h>
+#include <siri/db/server.h>
+#include <siri/db/buffer.h>
+#include <siri/buffersync.h>
+#include <uv.h>
+
+
+static uv_timer_t buffersync;
+
+#define BUFFERSYNC_INIT_TIMEOUT 1000
+
+static void BUFFERSYNC_cb(uv_timer_t * handle);
+
+void siri_buffersync_init(siri_t * siri)
+{
+    uint64_t repeat = (uint64_t) siri->cfg->buffer_sync_interval;
+    if (repeat == 0)
+    {
+        siri->buffersync = NULL;
+        return;
+    }
+    siri->buffersync = &buffersync;
+    uv_timer_init(siri->loop, &buffersync);
+    uv_timer_start(
+            &buffersync,
+            BUFFERSYNC_cb,
+            BUFFERSYNC_INIT_TIMEOUT,
+            repeat < 100 ? 100 : repeat);
+}
+
+void siri_buffersync_stop(siri_t * siri)
+{
+    if (siri->buffersync != NULL)
+    {
+        /* stop the timer so it will not run again */
+        uv_timer_stop(&buffersync);
+        uv_close((uv_handle_t *) &buffersync, NULL);
+        siri->buffersync = NULL;
+    }
+}
+
+
+static void BUFFERSYNC_cb(uv_timer_t * handle __attribute__((unused)))
+{
+    siridb_t * siridb;
+    llist_node_t * siridb_node;
+
+    siridb_node = siri.siridb_list->first;
+
+    while (siridb_node != NULL)
+    {
+        siridb = (siridb_t *) siridb_node->data;
+
+        /* flush the buffer, maybe on each insert or another interval? */
+        if (siridb_buffer_fsync(siridb))
+        {
+            log_critical("fsync() has failed on the buffer file");
+        }
+
+        siridb_node = siridb_node->next;
+    }
+}
+

diff --git a/src/siri/cfg/cfg.c b/src/siri/cfg/cfg.c
index be7a5dffd5c8bbb9655715184cb73758a25a51d3..520d9349f91e81f6ad19de2b2580ff730ca420f2 100644 (file)
--- a/src/siri/cfg/cfg.c
+++ b/src/siri/cfg/cfg.c
@@ -31,7 +31,8 @@ static siri_cfg_t siri_cfg = {
         .server_address="localhost",
         .default_db_path="/var/lib/siridb/",
         .pipe_support=0,
-        .pipe_client_name="siridb_client.sock"
+        .pipe_client_name="siridb_client.sock",
+        .buffer_sync_interval=0,
 };
 
 static void SIRI_CFG_read_uint(
@@ -130,6 +131,15 @@ void siri_cfg_init(siri_t * siri)
         SIRI_CFG_read_pipe_client_name(cfgparser);
     }
 
+    tmp = siri_cfg.buffer_sync_interval;
+    SIRI_CFG_read_uint(
+            cfgparser,
+            "buffer_sync_interval",
+            0,
+            300000,
+            &tmp);
+    siri_cfg.buffer_sync_interval = (uint32_t) tmp;
+
     cfgparser_free(cfgparser);
 }
 

diff --git a/src/siri/db/insert.c b/src/siri/db/insert.c
index 1d9921622c702f5fc0c0108d9170f9b601ee2f3e..34c9698b322507ad8672bef83ff7aab240c77a2f 100644 (file)
--- a/src/siri/db/insert.c
+++ b/src/siri/db/insert.c
@@ -13,6 +13,7 @@
 #include <logger/logger.h>
 #include <qpack/qpack.h>
 #include <siri/async.h>
+#include <siri/db/buffer.h>
 #include <siri/db/forward.h>
 #include <siri/db/insert.h>
 #include <siri/db/points.h>
@@ -953,6 +954,14 @@ static void INSERT_local_task(uv_async_t * handle)
         }
     }
 
+    if (siri.buffersync == NULL)
+    {
+        if (siridb_buffer_fsync(siridb))
+        {
+            log_critical("fsync() has failed on the buffer file");
+        }
+    }
+
     uv_mutex_unlock(&siridb->series_mutex);
     uv_mutex_unlock(&siridb->shards_mutex);
 

diff --git a/src/siri/heartbeat.c b/src/siri/heartbeat.c
index 6ee7dea0ccb1dabfebd60f37a776ffa705f7c14d..e398f499fe2dedab6fc45bf829d4d9192970dbc2 100644 (file)
--- a/src/siri/heartbeat.c
+++ b/src/siri/heartbeat.c
@@ -16,13 +16,8 @@
 #include <logger/logger.h>
 #include <siri/db/server.h>
 #include <siri/heartbeat.h>
-#include <siri/db/buffer.h>
 #include <uv.h>
 
-#if DEBUG
-#include <siri/db/series.h>
-#endif
-
 static uv_timer_t heartbeat;
 
 #define HEARTBEAT_INIT_TIMEOUT 1000
@@ -74,12 +69,6 @@ static void HEARTBEAT_cb(uv_timer_t * handle __attribute__((unused)))
     {
         siridb = (siridb_t *) siridb_node->data;
 
-        /* flush the buffer, maybe on each insert or another interval? */
-        if (siridb_buffer_fsync(siridb))
-        {
-            log_critical("fsync() has failed on the buffer file");
-        }
-
         server_node = siridb->servers->first;
         while (server_node != NULL)
         {

diff --git a/src/siri/siri.c b/src/siri/siri.c
index a73d39ce8c068702999e9b07bbc0acb24e3ea37e..9e59c296318bdc026d4b3329566ede162336f305 100644 (file)
--- a/src/siri/siri.c
+++ b/src/siri/siri.c
@@ -86,6 +86,7 @@ siri_t siri = {
         .fh=NULL,
         .optimize=NULL,
         .heartbeat=NULL,
+        .buffersync=NULL,
         .cfg=NULL,
         .args=NULL,
         .status=SIRI_STATUS_LOADING,
@@ -192,6 +193,9 @@ int siri_start(void)
     /* initialize heart-beat task (bind siri.heartbeat) */
     siri_heartbeat_init(&siri);
 
+    /* initialize buffer-sync task (bind siri.buffersync) */
+    siri_buffersync_init(&siri);
+
     /* initialize backup (bind siri.backup) */
     if (siri_backup_init(&siri))
     {
@@ -448,6 +452,9 @@ static void SIRI_signal_handler(
         /* stop heart-beat task */
         siri_heartbeat_stop(&siri);
 
+        /* stop buffer-sync task */
+        siri_buffersync_stop(&siri);
+
         /* destroy backup (mode) task */
         siri_backup_destroy(&siri);